package cn.lzd.demo1;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import scala.Tuple2;

import java.util.Arrays;

/**
 * Created by lzd on 2018/12/14.
 */
public class JavaWordCount {
    public static void main(String[] args) {
        SparkConf sparkConf = new SparkConf().setAppName("java-spark-wdCount").setMaster("local");
        JavaSparkContext sc = new JavaSparkContext(sparkConf);

        JavaRDD<String> lines = sc.textFile("/G:\\mr_profile\\spark\\wdcount.txt");

        JavaRDD<String> words = lines.flatMap((x) -> Arrays.asList(x.split(" ")).iterator());

        JavaPairRDD<Object, Integer> wordAndOne = words.mapToPair((x) -> new Tuple2<Object, Integer>(x, 1));

        JavaPairRDD<Object, Integer> reduce = wordAndOne.reduceByKey((x, b) -> x + b);

        //排序，java只支持sortBykey ，但是key是单词数不是次数

        //reduce.mapToPair((x)->x.swap());
        JavaPairRDD<Integer, Object> swaped = reduce.mapToPair((x) -> new Tuple2<Integer, Object>(x._2(), x._1()));

        //排序
        JavaPairRDD<Integer, Object> sorted = swaped.sortByKey(false);

        JavaPairRDD<Object, Integer> result = sorted.mapToPair((x) -> x.swap());

        result.saveAsTextFile("G:\\mr_profile\\spark\\out3");

        sc.stop();


    }
}
